/*
    Copyright 2006-2011 Patrik Jonsson, sunrise@familjenjonsson.org

    This file is part of Sunrise.

    Sunrise is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.

    Sunrise is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with Sunrise.  If not, see <http://www.gnu.org/licenses/>.

*/

/** \file 

    Implementations of threading functions. */

// $Id$

#include "threadlocal.h"
#include "config.h"
#include <iostream>
#include "mcrx-debug.h"
#include <sstream>

// This is for processor affinity on AIX
#ifdef HAVE_PROCESSOR_H
#include <sys/processor.h>
#endif
#ifdef HAVE_THREAD_H
#include <sys/thread.h> 
#endif

// Affinity on Linux
#ifdef HAVE_SCHED_SETAFFINITY
#include <sched.h>
#include <sys/syscall.h>
#include <errno.h>
#endif

// If we have no affinity API, then we need to define these
#if !defined(HAVE_SCHED_SETAFFINITY) && !defined(HAVE_SCHED_BINDPROCESSOR)
int errno;
#define EINVAL 0
#define ESRCH 0
#define EFAULT 0
#endif

using namespace std;

bool mcrx::bind_thread(int thread_number, int bank_size)
{ 
  const int cpu_number = 
    thread_number%bank_size + (thread_number/bank_size)*2*bank_size;

  int r=1;
#ifdef HAVE_BINDPROCESSOR
  // bind thread using AIX API
  r = bindprocessor (BINDTHREAD, thread_self (), cpu_number);
#else
#ifdef HAVE_SCHED_SETAFFINITY
  // bind thread using Linux API

  cpu_set_t global_cpuset;
  pid_t tid = (pid_t) syscall (SYS_gettid);
  stringstream s;

  // first find which cpus we are allowed to run on (necessary on
  // e.g. altix systems where we're running under a cpuset)
  //cout << "Getting current affinity. Valid cpus:\n";
  sched_getaffinity(tid, sizeof(cpu_set_t), &global_cpuset);
  int ntot=0;
  for(int i=0;i<CPU_SETSIZE; ++i) {
    if(CPU_ISSET(i, &global_cpuset)) {
      ++ntot;
    }
  }
  if(cpu_number==0) {
    s << "\nTotal " << ntot << " cpus in cpuset\n";
    cout << s.str();
  }

  if(cpu_number >= ntot) {
    cerr << "Error: Could not bind thread: " << thread_number
	 << " to cpu " << cpu_number
	 << " not enough cpus." << endl;
    return false;
  }
  else {
    // now we search up through the allowed list for our number
    cpu_set_t cpuset;
    CPU_ZERO(&cpuset);
    int i, n=-1;
    for(i=0; i<CPU_SETSIZE; ++i)
      if(CPU_ISSET(i,&global_cpuset)) {
	++n;
	if(n==cpu_number)
	  break;
      }
    s.str(""); s << "Binding thread " << thread_number  << " (TID " << tid <<") to cpu " << i << ".\n";
    cout << s.str();
    assert(CPU_ISSET(i, &global_cpuset));
    CPU_SET(i, &cpuset);
    assert(CPU_ISSET(i, &cpuset));

    // set affinity
    pid_t tid = (pid_t) syscall (SYS_gettid);
    r = sched_setaffinity(tid, sizeof(cpu_set_t), &cpuset);
    
    DEBUG(2,int olderr=errno;sched_getaffinity(tid, sizeof(cpu_set_t), &cpuset);printf("Thread %d (%d) has affinity %d\n",cpu_number, tid, cpuset);errno=olderr;);
  }
#endif
#endif
  if(r) {
    cerr << "Error: Could not bind thread " << cpu_number << endl;
    if (errno == EINVAL)
      cerr << "EINVAL" << endl;
    else if (errno == ESRCH)
      cerr << "ESRCH" << endl;
    else if (errno == EFAULT)
      cerr << "EFAULT" << endl;
    else
      cerr << "No affinity API available\n";
    return false;
  }
  return true;
}


#ifdef HAVE_LIBNUMA
#include <numa.h>
#endif


void mcrx::numa_info()
{
#ifdef HAVE_LIBNUMA
  assert(numa_available()!=-1);

  const int nn=numa_max_node();
  cout << "NUMA info: Allowed nodes:\n";
  for(int i=0; i<=nn; ++i)
    if(numa_bitmask_isbitset(numa_all_nodes_ptr,i)) {
      long mem=0, fr=0;
      mem = numa_node_size(i,&fr);
      printf("\tNode %d with %ld MB memory (%ld MB free)\n",i,mem>>20,fr>>20);
    }
  cout << "\n\tdistance matrix:\n\t";
  for(int i=0; i<=nn; ++i)
    if(numa_bitmask_isbitset(numa_all_nodes_ptr,i)) {
      for(int j=0; j<=nn; ++j)
	if(numa_bitmask_isbitset(numa_all_nodes_ptr,j))
	  cout << numa_distance(i,j) << '\t';
      cout << "\n\t";
    }
  cout << endl;
  return;
#endif
  cout << "No NUMA API defined\n";
}


bool mcrx::bind_allocations()
{
#ifdef HAVE_LIBNUMA
  numa_set_localalloc();
  return true;

#endif
  DEBUG(1,cerr << "Can't bind allocations: No NUMA memory policy API defined\n";);
  return false;
}


bool mcrx::interleave_allocations()
{
#ifdef HAVE_LIBNUMA
  cout << "Setting interleaving allocation policy\n";
  numa_set_interleave_mask(numa_all_nodes_ptr);
  return true;
#endif
  DEBUG(1,cerr << "Can't interleave allocations: No NUMA memory policy API defined\n";);
  return false;
}
